package com.otaku.crawler;

import java.io.IOException;
import java.util.List;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.w3c.dom.Node;
import org.xml.sax.SAXException;

import com.otaku.crawler.model.NASaveMoneyPojo;
import com.otaku.crawler.parser.NASaveMoneyParser;
import com.otaku.crawler.util.Constants;
import com.otaku.crawler.util.EmailHelper;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;

public class NASaveMoneyCrawler extends WebCrawler {
	private static final Logger log = Logger.getRootLogger();

	public boolean shouldVisit(WebURL url) {
		String href = url.getURL().toLowerCase();
		return !Constants.FILTERS.matcher(href).matches()
				&& href.startsWith("http://www.dealmoon.com/cn/")
				&& !href.endsWith(".html") && !href.contains("?sort=time")
				&& !href.startsWith("http://www.dealmoon.com/cn/coupons/")
				&& !href.contains("?action=")
				&& !href.startsWith("http://www.dealmoon.com/cn/deal-alert/")
				&& !href.endsWith("?s=guide")
				&& !href.startsWith("http://www.dealmoon.com/cn/malls");
	}

	public void visit(Page page) {
		String url = page.getWebURL().getURL();
		try {
			if (page.getParseData() instanceof HtmlParseData) {
				HtmlParseData htmlParseData = (HtmlParseData) page
						.getParseData();
				String html = htmlParseData.getHtml();

				Document d = Jsoup.parse(html);
				List<NASaveMoneyPojo> nal = NASaveMoneyParser.parse(d);

				log.info("url:" + url + "\nlist size:" + nal.size());
			}
		} catch (Exception e) {
			EmailHelper.sendMsg("NASaveMoneyCrawler caught exception in url:\n"
					+ url, e);
			e.printStackTrace();
			return;
		}
	}
}
